from scrapy import Spider, Request
from movie import items

class ygdySpider(Spider):
    name = 'dy'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
    }

    def start_requests(self):
        for i in range(1,41):
            yield Request(url='http://www.ygdy8.net/html/gndy/oumei/list_7_{}.html'.format(i),
                          headers=self.headers, callback=self.parse_menu)

    def parse_menu(self, response):
        title_info = response.xpath('//div[@id="header"]/div/div[3]/div[3]/div[2]/div[2]/div[2]/ul//td/b/a[2]/@href').extract()
        for one_movie in title_info:
            yield Request(url='http://www.ygdy8.net{}'.format(one_movie),
                          headers=self.headers, callback=self.parse_movie,
                          dont_filter=True)

    def parse_movie(self, response):
        # print response.status
        movie_url = response.xpath('//div[@id="Zoom"]//a/text()').extract()[0]
        movie = items.MovieItem()
        movie['movie_url'] = movie_url
        yield movie
